#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :place.py
# @Time      :2023/10/30 
# @Author    :CL
# @email     :1037654919@qq.com
import re

import requests
from bs4 import BeautifulSoup

from utils import mongo_manager
qunar_place = mongo_manager("qunar_place",db = 'lvyou')
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Pragma": "no-cache",
    "Proxy-Connection": "keep-alive",
    "Referer": "http://travel.qunar.com/p-oi715901-gulangyu",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
}
cookies = {
    "QN1": "0000c180306c57ae8cf8d525",
    "QN300": "s%3Dgoogle",
    "QN99": "8159",
    "QunarGlobal": "10.76.34.188_476f0ecd_18b7f08d2d0_-316c|1698646943873",
    "qunar-assist": "{%22version%22:%2220211215173359.925%22%2C%22show%22:false%2C%22audio%22:false%2C%22speed%22:%22middle%22%2C%22zomm%22:1%2C%22cursor%22:false%2C%22pointer%22:false%2C%22bigtext%22:false%2C%22overead%22:false%2C%22readscreen%22:false%2C%22theme%22:%22default%22}",
    "QN205": "s%3Dgoogle",
    "QN277": "s%3Dgoogle",
    "csrfToken": "bLP49THyP6qyD4ZpvJopVkZIYwMJQoeX",
    "_i": "DFiEuZkoc2SDZ7nLtuc9nvvIcDSw",
    "QN269": "B1AEDAD076EC11EEBD98FA163EC565DF",
    "QN601": "1844dcac9a8193dff4a2197440799d4c",
    "QN163": "0",
    "fid": "b781e0b7-f523-4a6c-b67d-c9266a2dc911",
    "QN48": "tc_f450c1fd5a4f469f_18b7f3f743c_83bc",
    "ariaDefaultTheme": "null",
    "QN57": "16986471185280.9089655108923953",
    "QN58": "1698647118527%7C1698647185294%7C4",
    "QN5": "qunar_djmp_gnmdd_%E6%AD%A6%E6%B1%89",
    "Hm_lvt_c56a2b5278263aa647778d304009eafc": "1698647830",
    "viewpoi": "9209153|715901",
    "uld": "1-299782-1-1698648417|1-368750-2-1698647842",
    "JSESSIONID": "8DF62F0D75B109F4C3E2572CC110BDB9",
    "QN267": "028366905ace069a3",
    "_vi": "VRLmNinoWFCCbuR6IHESbfNhHIneolNtK9rjNgiWFknD8IFPQaKILLTTYphafYIub13VrFpxUIx5AfzKO3bmVKmW4OwUV1c0Qoh6sUWODpi0iNWUgmKWBFVAe6ttuKpcM61H2DAYHkD3DtxL6XjZ_lPKPWsM8JIZyfkH3ZFhg9dR",
    "Hm_lpvt_c56a2b5278263aa647778d304009eafc": "1698648419",
    "QN271": "a7f896f7-8499-4685-b425-52cde20c4927",
    "SECKEY_ABVK": "NmQ3/dWPJQCKo0At1o15eCxrgbQMASA4VN6gWF+BwU4%3D",
    "BMAP_SECKEY": "NmQ3_dWPJQCKo0At1o15eN1G8blbELwpZ6E77aA7VdbBp8pmF_Q7qa-Z5ETQRpKgbvkFrir0JaFZsLZ_njMGN_yS2XvtiGOXs4Dr4k6xpT-kE95Ev953Xj6s-roo02hX1P5riBQAxASEoNeJ3YwOT5wo4LznhIUxATEERGKKIBYGWeMKYmmG3xtGikpmnj7V"
}
def get_place(url = "http://travel.qunar.com/place/"):

    response = requests.get(url, headers=headers, cookies=cookies, verify=False)

    # print(response.text)
    print(response)
    from bs4 import BeautifulSoup
    soups = BeautifulSoup(response.text, "lxml")
    datas = soups.find('div',class_ ='contbox current').find_all('dl')
    lists =[]
    for data in datas:
        hd = data.find('dt').get_text()

        for li in data.find_all('li'):
            lists.append({'topic':hd,'city':li.get_text(),'href':li.find('a').get('href')})
    return lists


if __name__ == "__main__":
    lists = get_place()
    # print(lists)
    print(len(lists))
    for data in lists:
        data["_id"] = data['href']
        if qunar_place.not_exist({"_id": data["_id"]}):
            url =  data['href'] +  '-jingdian'
            response = requests.get(url, headers=headers, cookies=cookies, verify=False)
            soup = BeautifulSoup(response.text, 'html.parser')
            # 使用find方法选择<p>元素
            try:
                p_element = soup.find('p', class_='nav_result')
                # 提取<p>元素中的文本
                text = p_element.get_text()
                # 使用正则表达式提取数字
                result = re.search(r'\((\d+)个\)', text)
                if result:
                    count = result.group(1)
                    print('count:', count)
                    data['count'] = count
            except:
                pass
            try:
                qunar_place.insertOne(data)
            except BaseException as e:
                print(e)
    qunar_place.close()
    # import pandas as pd
    # pd.DataFrame(lists,columns=['topic','city','href']).to_excel('qunar_place.xlsx')